package com.data.mining.project;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Date;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;

public class Indexer
{

	/**
	 * @param args
	 * @throws Exception
	 */
	public static void main( String[ ] args) throws Exception
	{
		if ( args.length != 2)
			throw new Exception( "Usage: java " + Indexer.class.getName( )
					+ " <index dir> <data dir>");

		File indexDir = new File( args[0]);
		File dataDir = new File( args[1]);

		long start = new Date( ).getTime( );
		int numIndexed = index( indexDir, dataDir);
		long end = new Date( ).getTime( );

		System.out.println( "Indexing " + numIndexed + " files took "
				+ ( ( end - start) / 60) + " seconds");

	}

	private static int index( File indexDir, File dataDir) throws IOException
	{
		if ( !dataDir.exists( ) || !dataDir.isDirectory( ))
		{
			throw new IOException( dataDir
					+ " does not exist os is not directory");
		}

		IndexWriter writer = new IndexWriter( indexDir,
				new StandardAnalyzer( ), true,
				IndexWriter.MaxFieldLength.UNLIMITED);
		writer.setUseCompoundFile( false);

		indexDirectory( writer, dataDir);

		int numIndexed = writer.docCount( );
		writer.optimize( );
		writer.close( );
		return numIndexed;
	}

	private static void indexDirectory( IndexWriter writer, File dataDir)
			throws IOException
	{
		for ( File file : dataDir.listFiles( ))
		{
			if ( file.isDirectory( ))
				indexDirectory( writer, file);
			else
				indexFile( writer, file);
		}
	}

	private static void indexFile( IndexWriter writer, File file)
			throws IOException
	{
		if ( file.isHidden( ) || !file.exists( ) || !file.canRead( ))
			return;

		System.out.println( "Indexing " + file.getCanonicalPath( ));

		Document document = new Document( );

		document.add( new Field( "content", new FileReader( file)));

		document.add( new Field( "filename", file.getCanonicalPath( ),
				Field.Store.YES, Field.Index.ANALYZED));
		writer.addDocument( document);
	}

}
